#!/bin/python3
# -*- coding: utf-8 -*-
import json
import os
import requests
import yaml
import time
import threading
import logging
import subprocess

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S'
                    )

WORKINGDIR = \
    IPTYPE = \
    CONFIG_NAME = \
    DONE_NAME = \
    GRUBIGNORE_NAME = ''

global_count = 0
sema = threading.Semaphore(10)


def pull_this(dirname):
    """同步指定文件夹下的所有任务.

    :param dirname:文件夹名 
    如果有一个文件下载失败则直接退出，否则会留下"Done"文件标记成功
    """
    with sema:
        global global_count
        logging.info('start to working on ' + dirname)

        # 避免切换工作目录 以便多线程
        config_name = os.path.join(dirname, CONFIG_NAME)

        try:
            with open(config_name) as file:
                the_list = json.load(file)
            for url in the_list:
                filename = os.path.join(dirname, url.rpartition('/')[2])
                #response = requests.get(url, stream=True, timeout=1)
                #total_size = int(
                #    response.headers['Content-Length']) / 1024 / 1024
                # logging.info('downloading file: ' + filename +
                #              ' total : %.2f Mb' % total_size)
                # with open(filename, 'wb') as file:
                #     for chunk in response.iter_content(chunk_size=1024):
                #         if chunk:
                #             file.write(chunk)
                if IPTYPE:
                    process = subprocess.Popen(['wget', '-O', os.path.join(WORKINGDIR,filename),
                                                '-t', '5', IPTYPE, url], stdout=subprocess.PIPE,
                                               stderr=subprocess.PIPE, cwd=dirname)
                else:
                    process = subprocess.Popen(['wget', '-O', os.path.join(WORKINGDIR,filename),
                                                '-t', '5', url], stdout=subprocess.PIPE, 
                                                stderr=subprocess.PIPE, cwd=dirname)
                if process.wait():
                    logging.error("failed download file: {}".format(filename))
                    logging.error(process.stderr.read().decode())
                    continue
                logging.info('finish downloading file: ' + filename)
        except:
            logging.warning('haven\'t done but have to leave by ' + filename)
        else:
            global_count += 1
            logging.info('all the work have done in ' + filename)
            with open(os.path.join(dirname, DONE_NAME), 'w') as file:
                file.write(str(time.localtime(time.time())))


def pull():
    """Pull所有的任务.

    支持多线程
    """
    global global_count
    os.chdir(WORKINGDIR)
    dirlist = os.listdir(WORKINGDIR)
    # 剔除ignore
    IGNORELIST = set()
    if os.path.isfile(GRUBIGNORE_NAME):
        with open(GRUBIGNORE_NAME) as file:
            IGNORELIST = set(json.load(file))
    pulllist = []
    for dir in dirlist:
        # 也可以这样
        # if dir in IGNORELIST:
        #     continue
        if os.path.isdir(dir):
            if os.path.exists(dir + '/' + DONE_NAME):
                logging.info(dir + DONE_NAME +
                             ' does exists, nothing touched!')
            else:
                if os.path.isfile(dir + '/' + CONFIG_NAME):
                    if dir not in IGNORELIST:
                        pulllist.append(dir)
                else:
                    logging.info(dir + CONFIG_NAME + ' doesn\'t exists!, quit')
        else:
            logging.info(dir + ' is not a directory,jump!')

    length_of_pulllist = len(pulllist)
    logging.info('total : ' + str(length_of_pulllist))

    Threads = []
    for filename in pulllist:
        t = threading.Thread(target=pull_this, args=(filename,))
        Threads.append(t)
    for i in range(length_of_pulllist):
        Threads[i].start()
    for i in range(length_of_pulllist):  # wait for all
        Threads[i].join()  # threads to finish
    logging.info('all DONE')
    logging.info(str(global_count) + ' / ' +
                 str(length_of_pulllist) + 'in total')
    ret = global_count
    global_count = 0
    return ret


def init(config_file):
    """init,must use before any function."""
    global WORKINGDIR, CONFIG_NAME, DONE_NAME, GRUBIGNORE_NAME, IPTYPE
    with open(config_file) as file:
        dic = yaml.load(file.read())
    WORKINGDIR = dic['WORKINGDIR']
    CONFIG_NAME = dic['CONFIG_NAME']
    DONE_NAME = dic['DONE_NAME']
    GRUBIGNORE_NAME = dic['GRUBIGNORE_NAME']
    if dic['IPTYPE'] == 'ipv6': IPTYPE = '-6'
    elif dic['IPTYPE'] == 'ipv4': IPTYPE = '-4'
    else: IPTYPE = ''
    return config_file


if __name__ == "__main__":
    init('config.yaml')
    if WORKINGDIR == '':
        WORKINGDIR = input('please enter working directory')
    pull()
